import operator

import feedparser
from naivebayes.bayes import *


def calc_most_freq(vocablist, full_text):
    print(vocablist)
    import operator
    freq_dict = {}
    for token in vocablist:
        freq_dict[token] = full_text.count(token)
    sorted_freq = sorted(freq_dict.items(), key=operator.itemgetter(1), reverse=True)
    return sorted_freq[:30]


def local_words(feed1, feed0):
    import feedparser
    doc_list = []
    class_list = []
    full_text = []
    len0 = len(feed0['entries'])
    len1 = len(feed1['entries'])
    min_len = len0 if len0 < len1 else len1
    for i in range(min_len):
        word_list = text_parse(feed1['entries'][i]['summary'])
        doc_list.append(word_list)
        full_text.extend(word_list)
        class_list.append(1)
        word_list = text_parse(feed0['entries'][i]['summary'])
        doc_list.append(word_list)
        full_text.extend(word_list)
        class_list.append(0)
    vocab_list = create_vocab_list(doc_list)
    print(vocab_list)
    top30_words = calc_most_freq(vocab_list, full_text)
    for pair_w in top30_words:
        if pair_w[0] in vocab_list:
            vocab_list.remove(pair_w[0])
    train_set = list(range(2*min_len))
    test_set = []
    for i in range(20):
        rand_index = int(random.uniform(0, len(train_set)))
        test_set.append(rand_index)
        del(train_set[rand_index])
    train_matri = []
    train_class = []
    for doc_index in train_set:
        train_matri.append(bag_of_words2_vec(vocab_list, doc_list[doc_index]))
        train_class.append(class_list[doc_index])
    p0_vec, p1_vec, p_spam = train_nb0(array(train_matri), array(train_class))
    error_count = 0
    for doc_index in test_set:
        word_vec = bag_of_words2_vec(vocab_list, doc_list[doc_index])
        if classify_nb(word_vec, p0_vec, p1_vec, p_spam) != \
                class_list[doc_index]:
                error_count += 1
    print('The error is: ', error_count/len(test_set))
    return vocab_list, p0_vec, p1_vec


def get_top_words(ny, sf):
    import operator
    vocab_list, p0_vec, p1_vec = local_words(ny, sf)
    top_ny = []
    top_sf = []
    for i in range(len(p0_vec)):
        if p0_vec[i] > -6.0 :
            top_sf.append((vocab_list[i], p0_vec[i]))
        if p1_vec[i] > -6.0 :
            top_ny.append((vocab_list[i], p1_vec[i]))
    sorted_sf = sorted(top_sf, key=lambda pair:pair[1], reverse=True)
    print("SF*SF*SF")
    for item in sorted_sf:
        print(item[0])
    sorted_ny = sorted(top_ny, key=operator.itemgetter(1), reverse=True)
    print("NY*NY*NY")
    for item in sorted_ny:
        print(item[0])


ny = feedparser.parse('http://newyork.craigslist.org/stp/index.rss')
sf = feedparser.parse('http://sfbay.craigslist.org/stp/index.rss')

get_top_words(ny, sf)

voc_list = create_vocab_list(['A', "b", 'C', 'A'])
print(voc_list)

test_dict = {'A':'3', 'B':'4', 'C':'2'}
sorted_dict = sorted(test_dict.items(), key=operator.itemgetter(1), reverse=True)
for item in sorted_dict:
    print(item[0])
